{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Reinforcement Learning in Action\n",
    "### by Alex Zai and Brandon Brown\n",
    "\n",
    "#### Chapter 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_action(actions):\n",
    "    best_action = 0\n",
    "    max_action_value = 0\n",
    "    for i in range(len(actions)):\n",
    "        cur_action_value = get_action_value(actions[i])\n",
    "        if cur_action_value > max_action_value:\n",
    "            best_action = i\n",
    "            max_action_value = cur_action_value\n",
    "    return best_action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import stats\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    " \n",
    "n = 10\n",
    "probs = np.random.rand(n)\n",
    "eps = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_reward(prob, n=10):\n",
    "    reward = 0\n",
    "    for i in range(n):\n",
    "        if random.random() < prob:\n",
    "            reward += 1\n",
    "    return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.9655"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([get_reward(0.7) for _ in range(2000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "record = np.zeros((n,2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_record(record,action,r):\n",
    "    new_r = (record[action,0] * record[action,1] + r) / (record[action,0] +\n",
    "     1)\n",
    "    record[action,0] += 1\n",
    "    record[action,1] = new_r\n",
    "    return record"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_arm(record):\n",
    "    arm_index = np.argmax(record[:,1],axis=0)\n",
    "    return arm_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1a14a93810>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZW0lEQVR4nO3df3Rc5X3n8ffHsowlSBE/1BQEjgNhTUmgNiiJs3S7gTQ1TUNCSShQaJqUXXbP/ijkdNXFp3RJzyZlW2UJu6cpJ4ak2S1ekgJmQtltFA6BJu2CG9kyEQbUGJYfHtPiACIBi1jI3/3j3rFlaWY0kuZqZu58XufoSHPv9dznEeKjR8/z3OdRRGBmZvmzrNEFMDOzbDjgzcxyygFvZpZTDngzs5xywJuZ5dTyRhdguuOPPz5Wr17d6GKYmbWMbdu2/TAiesuda6qAX716NcPDw40uhplZy5D0bKVz7qIxM8spB7yZWU454M3McsoBb2aWUw54M7OccsCbmeVUU02TNLPmUBgpsnHL95mYPFDT9cd0d3LDhe/konV9C36PVrRM8OvvXcVnLzqT6wujbH7kORa6Pu/096oXNdNywf39/eF58GZzK4wU+cy9OxmfmDx4rLsz+YN8X44DtR1cuX5+IS9pW0T0lzvnFrzZEqvU0iu1goFZ4V0LB3s+3LH1+bq14h3wlmszW7rluhJq/bfTdXcu44jODl7ZN78QruaVfZNc+/UddXs/a01TdexVyTTgJX0a+BdAAKPApyLijSzvae2rWiCXlEJ0sUG6b/KAW8yWiQ6pbu+VWcBL6gN+GzgjIiYk/QVwGfDVrO5p+TJzkG7mIFQ7DOJZ+7n8vSfX7b2y7qJZDnRJmgS6gT0Z38+aQC0t6RUdYv/U/P4UPRBw+yPPcfsjzy22iGZNJ4tZNJkFfEQUJX0eeA6YAL4VEd/K6n7WOAtpSc833K0xBFyRzuoojBQZHBpjz/gEJ/Z0cd7pvdy9bfec/92nvwcw630GNqypaUykmVSbEplFUC9UZtMkJR0D3A1cCowDdwJ3RcTtM667GrgaYNWqVec8+2zFlS9tCV1fGOWOrc8zFUGHxOXvPfng/+QLmeFhjTF9UDkPwWqzVZsmmWXAXwJcEBFXpa8/AayPiH9T6d94HnxjLPYBDVsYkcw+KH2uZD4zf6z9NGoe/HPAekndJF00HwCc3g3iAcnFWdEhjjxiecVpkdP/yjFrFln2wW+VdBewHXgTGAE2ZXU/O+T6wqgHInHL1yzTWTQRcQNwQ5b3sEPyHOwdgkrjsjMH8cws4SdZl8DMPu4jV3TwuV89s64tyw/e9BA/ePH1ur1fvUyva7luoiy+F2aW8GJjGbvi1of526deLntu5uJQC+lSKIwUGbhzB83Ste5uEbOl1ZBZNAuRt4BfTJdJqWU7/OzLZacrNkN3jMPcrPEc8A1QGClmtnDUyg7xRkYPCh25ooN9+6cOPshy36MvLGihLjNbGl4uuAH+493fz+y9FxPu8x2Q9MClWetywNdZM/WJL18mPn/Jz7nFbdamHPB11Az94iXnnnosm//l+xpdDDNrIG+6XSfNFO5Xrl/lcDczt+DroTBSbIpw9yComU3ngK+D37tntKbrzj31WC7pX3XYaoylrd/G901ydFcnP3pjkgPzHEN1d4yZleOAX6TCSJHX90/Ned30EK7Wwi63pCvAwF2PMjlj9swRy5fxRx87yy12MyvLAb9ItbTer5zHtMSL1vVVDGyv5W1m8+GAX4Qrbn24autdwBcuXVuXIK4W/GZm5XgWzQJVW2OmpF7hbma2EA74BSiMFOcM967OZQ53M2soB/wC1NLvfuPFZy1BSczMKnPAz1Mts2auXL/KrXczazgH/Dz9wV/urHr+3FOP9QJdZtYUHPDzUBgpVtx0GaBzGX7gyMyahgN+Hubqex+8ZO0SlcTMbG4O+BrN1ffufnczazYO+BrN1Xp3v7uZNRsHfA3mar33dHUuYWnMzGrjgK/BXDNnPvORdy5RSczMaueAn8NcM2f8xKqZNSsH/Bzmar37iVUza1YO+DlUa7175oyZNTMHfBWFkWLV8545Y2bNzOvBz3B9YZQ7tj7PVFTfN88zZ8ys2Tngp7m+MFrz5tmeOWNmzc5dNNPcsfX5mq4T1fdVNTNrBg74aebqlimp7Sozs8ZywKfmGlCdrq+nK8OSmJnVhwM+Ndd89+kGNqzJsCRmZvXhgGfup1Wn6+nqdP+7mbUEBzy17bFa4tkzZtYq2j7gr7j14Tn3WC3p9rozZtZC2jrgCyNF/vapl2u6trND/KHXnTGzFtLWDzrVOrDa19PFwIY1br2bWUvJNOAl9QC3Ae8imT7+WxHxcJb3nI+5BlZvvnStQ93MWlbWLfj/BnwzIj4uaQXQnfH96sYrRZpZq8usD17STwG/AHwZICL2R8R4VvdbiO7O8tXvXOaVIs2s9WU5yHoKsBf4M0kjkm6TdGSG95uX6wuj7Js8MOv4MsHgJWsbUCIzs/rKMuCXA2cDt0TEOuB14LqZF0m6WtKwpOG9e/dmWJxDCiNFNldYNfKnVvpBJjPLhywDfjewOyK2pq/vIgn8w0TEpojoj4j+3t7eDItzyODQWMUFw16dqO2JVjOzZpdZwEfEPwDPSyot3PIB4PGs7jcfe8YnKp470QuJmVlOZD2L5t8Dm9MZNE8Dn8r4flUVRopVW+/CC4mZWX5kGvARsQPoz/IetSqMFNm4ZZSJycrLElzhqZFmliNts1TB4NBY1XDvXCb633bsEpbIzCxbbRPw1frdASYPBINDY0tUGjOz7LVNwNcyeDrXLwEzs1bSNgE/sGENmuMaz6Axszxpm4CHuTfL9gwaM8uTtgj4wkiRgTsfrXpNlzfzMLOcaYuAHxwaY/JA5fb7MsGN3szDzHKmLQJ+rsHTm37N676bWf60RcBXGzzt6+lyuJtZLrVFwA9sWMOyMlNoOjvkgVUzy622CHiADs1O+EvffbJb72aWW20R8JUGWR98cmnWnzcza4SKi41JqrowS0S8XP/iZKPSIKufXDWzPKu2muQ2kmeDBKwCXkm/7gGeA96eeenqpKe7k1f2zd7Iw0+umlmeVeyiiYi3R8QpwBBwYUQcHxHHAR8GtixVARerMFLktTfenHXcA6xmlne19MG/OyL+T+lFRPwV8M+zK1J9Vep/P3LFcg+wmlmu1bLhxw8lXQ/cTtJlcyXwUqalqqNK/ezee9XM8q6WFvzlQC9wT/rRmx5rCZX62d3/bmZ5VzXgJXUAGyPimohYFxFnR8S1rTSDZmDDGro6Ow471tXZ4f53M8u9qgEfEVPAOUtUlswcsfxQNY/p7uTGi890/7uZ5V4tffAjku4F7gReLx2MiKafSVNuo+03Jg80sERmZkunlj74Y0kGVc8HLkw/Ppxloeql3EbbE5NT3nvVzNrCnC34iPjUUhQkC36C1cza2ZwBL2klcBXwTmBl6XhE/FaG5aqLSk+w9nR3NqA0ZmZLq5Yumj8HfgbYAPw1cBLw4ywLVS9RYROnSsfNzPKkloB/R0T8PvB6RPwP4FeAM7MtVn2MV3iYyQ85mVk7qCXgS2k4LuldwNHA6sxKVCfXF0YrnvNDTmbWDmqZJrlJ0jHA7wP3AkelXzetwkiRzY88V/acwA85mVlbqGUWzW3pl38NnJJtcepjcGiMSt3sAX7IyczaQi2zaJ4CHgG+C3wnIh7PvFSLVG0aZJ+7Z8ysTdTSB38G8CXgOODzkp6WdE+2xVqcSn3s7p4xs3ZSS8BPkQy0TgEHgH8EXsyyUItVboExAVesX+XuGTNrG7UMsv4IGAVuAm6NiKZfC74U4oNDY+wZn+DEni4GNqxxuJtZW1HM8dSPpI8CPw+8B9gP/F+SvvgH6l2Y/v7+GB4ervfbmpnllqRtEdFf7lwts2i+AXxD0unALwPXAr8LeLTSzKyJ1TKL5m5gLbCLZCbNJ4CtGZdr0QojRXfRmFlbq6UP/r8A29PNP1rCzHXgi+MTbNySPNnqkDezdlHLLJqdwEZJmwAknSapqdeD9zrwZma1BfyfkQyu/tP09W7gs5mVqA68DryZWW0Bf2pE/DHpomMRMUEyrbwmkjokjUi6b4FlnLdKDzp5kTEzaye1BPx+SV0ky7gg6VTgJ/O4xzXAEwso24Kdd3rvrN9AXZ0dforVzNpKLQF/A/BN4GRJm4EHSKZJzknSSSTrx98217X1Uhgpcve24mGLjQn42Dl9HmA1s7ZSyzz4+yVtB9aTZOU1EfHDGt//ZpJfBm9ZeBHnp9wAawAPPrl3qYpgZtYUamnBExEvRcT/joj7gOMk3TrXv0ln2rwYEdvmuO5qScOShvfuXXwIe4DVzCxRMeAlnSXpW5Iek/RZSW9NH3p6AKhlyeBzgY9Iegb4GnC+pNtnXhQRmyKiPyL6e3t7F1iNQzzAamaWqNaCvxX4X8DHgL3AduBpkj1avzDXG0fExog4KSJWA5cB346IKxdf5OoGNqyhs+PwIdbODnmA1czaTrU++CMi4qvp12OS/gNwXUs80Tpz/bTq66mZmeVStYBfKWkdh+a8vwacJUkAEbG91ptExEPAQwss47wMDo0xeeDwRJ88EAwOjXkWjZm1lWoB/wLJGvAl/zDtdQDnZ1WoxfAgq5lZomLAR8R5S1mQeunp7uSVfZOzjnuQ1czaTU3TJFtFYaTIa2+8Oeu4B1nNrB3lKuDL9b8DHLliufvfzazt5CrgK/Wzvzoxu8vGzCzvatnR6ewyh18Fno2I2f0hDXRiTxfFMiHv/ncza0e1tOD/FHgE2ETy8NPDJE+m/r2kX8qwbPM2sGENXZ0dhx3zKpJm1q5qCfhngHXpcgLnAOuAx4BfBP44w7LN20Xr+rjx4jPp6+lCQF9PFzdefKb7382sLdWyJ+vpEbGz9CIiHpe0LiKeTp95aioXrfOywGZmUFvAj0m6haRbBuBSku6ZI0h3eTIzs+ZTSxfNJ4FdwLXAp0kWHPskSbi35MNQZmbtoJYW/AXAn0TEfy1z7rU6l8fMzOqkloD/CHCzpO+QdNMMNdv0yJLCSJHBoTH2jE9wYk8XAxvWuD/ezNrWnF00EfEp4B3AncCvA09JWrI9VmtVGCmyccsoxfEJAiiOT7BxyyiFkWKji2Zm1hC1btk3CfwVSQt+G/DRLAu1EOX2Yp2YnGJwaKxBJTIza6w5A17SBZK+SjLQ+nHgNuCEjMs1b14m2MzscLX0wX+SpOX+ryLiJ9kWZ+G8TIGZ2eFq6YO/LCIKpXCXdK6kL2ZftPnxMgVmZoerpQWPpLUkA6y/Bvw/YEuWhVqI0mwZz6IxM0tUDHhJ/wS4DLgceAn4OqBm3unJyxSYmR1SrQX/JPBd4MKI2AUg6dNLUiozM1u0an3wHyPZaPtBSbdK+gDQfKuLmZlZWRUDPiLuiYhLgdOBh0jWoXmrpFuabR14MzObrZZZNK9HxOaI+DBwErADuC7zkpmZ2aLMa0/WiHg5Ir4UEednVSAzM6uPXG26bWZmhzjgzcxyygFvZpZTDngzs5xywJuZ5ZQD3swspxzwZmY55YA3M8spB7yZWU454M3McsoBb2aWUw54M7OccsCbmeWUA97MLKcyC3hJJ0t6UNITknZKuiare5mZ2WzV9mRdrDeB34mI7ZLeAmyTdH9EPJ7VDQsjRQaHxtgzPsGJPV0MbFjjTbjNrG1lFvAR8QLwQvr1jyU9AfQBmQR8YaTIxi2jTExOAVAcn2DjllEAh7yZtaUl6YOXtBpYB2zN6h6DQ2MHw71kYnKKwaGxrG5pZtbUMg94SUcBdwPXRsSPypy/WtKwpOG9e/cu+D57xifmddzMLO8yDXhJnSThvjkitpS7JiI2RUR/RPT39vYu+F4n9nTN67iZWd5lOYtGwJeBJyLipqzuUzKwYQ1dnR2HHevq7GBgw5qsb21m1pSybMGfC/wGcL6kHenHh7K62UXr+rjx4jPp6+lCQF9PFzdefKYHWM2sbWU5i+ZvAGX1/uVctK7PgW5mlvKTrGZmOeWANzPLKQe8mVlOOeDNzHIqy7VoloTXnzEzK6+lA97rz5iZVdbSXTRef8bMrLKWDnivP2NmVllLB7zXnzEzq6ylA97rz5iZVdbSg6ylgVTPojEzm62lAx68/oyZWSUt3UVjZmaVOeDNzHLKAW9mllMOeDOznGr5QVZIliz4zL07GZ+YBOCY7k5uuPCdHnw1s7bW8gFfGCkycOejTB6Ig8de2TfJwF2PAl6TxszaV8t30QwOjR0W7iWTU+E1acysrbV8wFdbd8Zr0phZO2v5gK+27ozXpDGzdtbyAT+wYU3FSpx3eu+SlsXMrJm0fMBftK6Po7s7y5578Mm9S1waM7Pm0fIBDzC+b7LscffBm1k7y0XAe114M7PZchHwXhfezGy2ln/QCbwuvJlZObkIePC68GZmM+Wii8bMzGZzwJuZ5ZQD3swspxzwZmY55YA3M8spB7yZWU454M3McsoBb2aWUw54M7OccsCbmeWUA97MLKcyDXhJF0gak7RL0nVZ3svMzA6X2WJjkjqALwIfBHYD35N0b0Q8Xs/7FEaKXkXSzKyMLFvw7wF2RcTTEbEf+Brw0XreoDBSZOOWUYrjEwRQHJ9g45ZRCiPFet7GzKwlZRnwfcDz017vTo/VzeDQGBOTU4cdm5icYnBorJ63MTNrSVkGvMoci1kXSVdLGpY0vHfv/DbJrrTnqvdiNTPLNuB3AydPe30SsGfmRRGxKSL6I6K/t7d3XjfwXqxmZpVlGfDfA06T9HZJK4DLgHvreQPvxWpmVllms2gi4k1J/w4YAjqAr0TEznrew3uxmplVpohZ3eIN09/fH8PDw40uhplZy5C0LSL6y53zk6xmZjnlgDczyykHvJlZTjngzcxyygFvZpZTTTWLRtJe4NkF/vPjgR/WsTitwHXOv3arL7jO8/W2iCj7lGhTBfxiSBquNFUor1zn/Gu3+oLrXE/uojEzyykHvJlZTuUp4Dc1ugAN4DrnX7vVF1znuslNH7yZmR0uTy14MzObxgFvZpZTLR/wki6QNCZpl6TrGl2eepH0FUkvSnps2rFjJd0v6Qfp52PS45L039Pvwfclnd24ki+cpJMlPSjpCUk7JV2THs9tvSWtlPR3kh5N6/wH6fG3S9qa1vnr6Z4KSDoifb0rPb+6keVfKEkdkkYk3Ze+znV9ASQ9I2lU0g5Jw+mxTH+2WzrgJXUAXwR+GTgDuFzSGY0tVd18FbhgxrHrgAci4jTggfQ1JPU/Lf24GrhlicpYb28CvxMRPwusB/5t+t8zz/X+CXB+RPwcsBa4QNJ64I+AL6R1fgW4Kr3+KuCViHgH8IX0ulZ0DfDEtNd5r2/JeRGxdtqc92x/tiOiZT+A9wFD015vBDY2ulx1rN9q4LFpr8eAE9KvTwDG0q+/BFxe7rpW/gC+AXywXeoNdAPbgfeSPNW4PD1+8OecZAOd96VfL0+vU6PLPs96npSG2fnAfST7N+e2vtPq/Qxw/Ixjmf5st3QLHugDnp/2end6LK/eGhEvAKSffzo9nrvvQ/qn+DpgKzmvd9pdsQN4EbgfeAoYj4g300um1+tgndPzrwLHLW2JF+1m4HeBA+nr48h3fUsC+JakbZKuTo9l+rOd2ZZ9S0RljrXjvM9cfR8kHQXcDVwbET+SylUvubTMsZard0RMAWsl9QD3AD9b7rL0c0vXWdKHgRcjYpuk95cOl7k0F/Wd4dyI2CPpp4H7JT1Z5dq61LvVW/C7gZOnvT4J2NOgsiyFf5R0AkD6+cX0eG6+D5I6ScJ9c0RsSQ/nvt4AETEOPEQy/tAjqdQAm16vg3VOzx8NvLy0JV2Uc4GPSHoG+BpJN83N5Le+B0XEnvTziyS/yN9Dxj/brR7w3wNOS0fgVwCXAfc2uExZuhf4zfTr3yTpoy4d/0Q68r4eeLX0Z18rUdJU/zLwRETcNO1UbustqTdtuSOpC/hFksHHB4GPp5fNrHPpe/Fx4NuRdtK2gojYGBEnRcRqkv9fvx0RV5DT+pZIOlLSW0pfA78EPEbWP9uNHniow8DFh4C/J+m3/L1Gl6eO9boDeAGYJPltfhVJ3+MDwA/Sz8em14pkNtFTwCjQ3+jyL7DOP0/yZ+j3gR3px4fyXG/gLGAkrfNjwH9Kj58C/B2wC7gTOCI9vjJ9vSs9f0qj67CIur8fuK8d6pvW79H0Y2cpq7L+2fZSBWZmOdXqXTRmZlaBA97MLKcc8GZmOeWANzPLKQe8mVlOOeCtLUmaSlf1e0zSnZK60+OvNbpsZvXigLd2NRHJqn7vAvYD/7rRBTKrNwe8GXwXeMf0A5KOkvSApO3pGt4fTY//59I69enrz0n6bUknSPrOtL8K/tkS18FsFj/oZG1J0msRcVS6vsndwDcj4pYZx7sjWezseOARkrW53wZsiYizJS0jeQLxPcAngZUR8bl0n4LuiPhxQypnlmr11STNFqorXaIXkhb8l2ecF/CHkn6BZFnbPpKlXZ+R9JKkdcBbgZGIeEnS94CvpIulFSJiB2YN5oC3djUREWurnL8C6AXOiYjJdPXDlem520ha7D8DfAUgIr6T/jL4FeDPJQ1GxP/MqvBmtXAfvFl5R5OsWz4p6TySrpmSe0i2U3w3yY5DSHpbev2tJH8NtNz+sJY/bsGblbcZ+Mt0c+QdwMHNGSJiv6QHSXYhmkoPvx8YkDQJvAZ8YonLazaLB1nN5ikdXN0OXBIRP2h0ecwqcReN2TxIOoNkbfIHHO7W7NyCNzPLKbfgzcxyygFvZpZTDngzs5xywJuZ5ZQD3swsp/4/vNh/WZxEpEUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,1)\n",
    "ax.set_xlabel(\"Plays\")\n",
    "ax.set_ylabel(\"Avg Reward\")\n",
    "record = np.zeros((n,2))\n",
    "probs = np.random.rand(n)\n",
    "eps = 0.2\n",
    "rewards = [0]\n",
    "for i in range(500):\n",
    "    if random.random() > eps:\n",
    "        choice = get_best_arm(record)\n",
    "    else:\n",
    "        choice = np.random.randint(10)\n",
    "    r = get_reward(probs[choice])\n",
    "    record = update_record(record,choice,r)\n",
    "    mean_reward = ((i+1) * rewards[-1] + r)/(i+2)\n",
    "    rewards.append(mean_reward)\n",
    "ax.scatter(np.arange(len(rewards)),rewards)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(av, tau=1.12):\n",
    "    softm = np.exp(av / tau) / np.sum( np.exp(av / tau) ) \n",
    "    return softm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.arange(10)\n",
    "av = np.zeros(10)\n",
    "p = softmax(av)\n",
    "np.random.choice(x,p=p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1a15619f90>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAE9CAYAAAA/N6mDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAYqElEQVR4nO3df5RfdX3n8efbSYABWsOPkZKABoWNVVSCUw8tuz0KarBQoVu2hUoV113Ws90tcGo8ZI891K3sdpuudve0pQ2USpVWFMJI3dbIoVhoT2GdZKLh16xIEZmgGYFR0akM4b1/fO/gMCSTbzJz7/3MfJ+Pc3Lmez/fy/e+/cTzzWvu/fyIzESSJKkEL2m7AEmSpGkGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxVjWdgHdOProo3P16tVtlyFJkhbA1q1bv52ZA3t6r7ZgEhHXAecAuzLz5KrtSOBGYDXwCPBLmfnUvj5r9erVDA8P11WqJElqUER8fW/v1fko5+PAWbPargBuz8yTgNurY0mSJKDGYJKZdwJPzmo+F7i+en09cF5d15ckSYtP04Nfj8nMxwGqny/b24kRcUlEDEfE8Pj4eGMFSpKk9hQ7KyczN2XmYGYODgzscXyMJElaYpoOJt+KiGMBqp+7Gr6+JEkqWNPB5FbgPdXr9wCfbfj6kiSpYLUFk4j4S+AfgTUR8VhEvA/4HeBtEfFV4G3VsSRJElDjOiaZeeFe3jqzrmtKkqTFbVGs/Cppz4ZGxti4ZZSxiUkCyC7+myMOXc6VP/9azlu7aq+ft3NikpUr+lm/bg0Av3XrfUxMTi1s8ZKKN9f3RV0is5uvsnYNDg6mK7+qDR8a2sENdz/a1T/4krQULe8LNp7/hgUNJxGxNTMH9/Sed0x0wGb+tt4Xwe5MVlW/ZTeVrodGxvxtXpJqNLU72bhltLHvdYOJ5jQ0MsaGzV9hcuq5Oc/bXd15G5uY5LIbt3PZjdv3eu4Rhy7n7Ncfy+e+/LiBQpIWgZ0Tk41dy2CyxCyGOwhP/WCKT979aNtlSJK6tHJFf2PXMpgscoshiEiSFq/lffH8QPgmGExadiCzKiRJakIbs3IMJi161zX/yD987UcbMBtKtBD29kXS7XihvTnsoD6u+oXXNfoFJan3GEwaNPPuiBaXNn5rWGjnrV21qOuX1BsMJg2Y72+qpXtJwHMt3+5ZCsFBkmQwqd2HhnYsiRkoC/EP/94G6k5PH77jwfEXrDhqyJCk3mMwqUEpM2VKu4vgowRJ0r4YTBbY7AGtTSkthEiSdCAMJvO00HdHVs3YOG32ZmqGDknSUmcwOUALOaB1b9MwDSKSpF5jMDkA8x3QOr2QWtMb3kmSVDqDyX4aGhnjhnmEkotOezkfOe91C1iRJElLh8FkPwyNjHH5p7cf0AqtDk6VJGnfDCZdGhoZY/1nvkzuZyo5/VVHcsO//+l6ipIkaYkxmOzDgS4jH8C7fGwjSdJ+MZjMoTPzZgeTU7v3ee5BfcFhBy9j4gdTTu+VJOkAGUzm8OG/uq+rULKifznbr3x7AxVJkrS0vaTtAkr1oaEdPPWDfS+atrwv+K13vraBiiRJWvq8YzLL/i6ctvH8N/jIRpKkBWIwmWF65s3Uc91NvVnRv9xQIknSAvJRzgwbt4x2HUoAH+FIkrTAvGMyw84upwRPTwX2bokkSQvLYDLDyhX9+1yvxP1tJEmqj8Fkhre8emCvm/O5x40kSfUzmFSGRsa4eevYi9oPO6iPq37hdd4hkSSpAQ5+rWzcMrrHxdRWHHqQoUSSpIYYTCp7G/ja7YBYSZI0fwaTysoV/fvVLkmSFp7BpLJ+3Rr6l/e9oK1/eR/r161pqSJJknqPwWSGg5f9qDuOOHQ5//1fO+hVkqQmOSuH6f1xdrxg8Os/d7lXjiRJWjjeMWHPM3Imp3azcctoSxVJktSbDCY4I0eSpFIYTHBGjiRJpTCY4IwcSZJK4eBXeH7mzcYto+ycmGSlG/VJktQKg0nlvLWrDCKSJLWslWASEZcD/w5IYAfw3sz85zZqgc50Ye+WSJLUvsbHmETEKuDXgcHMPBnoAy5ouo5p02uYjE1MksDYxCQbNu9gaOTFOw1LkqR6tTX4dRnQHxHLgEOBnS3V4RomkiQVpPFgkpljwO8BjwKPA9/JzC/MPi8iLomI4YgYHh8fr60e1zCRJKkcbTzKOQI4FzgBWAkcFhEXzT4vMzdl5mBmDg4MDNRWj2uYSJJUjjYe5bwV+KfMHM/MKWAz8DMt1AG4hokkSSVpY1bOo8BpEXEoMAmcCQy3UAfgGiaSJJWk8WCSmfdExE3ANuBZYATY1HQdM7mGiSRJZWhlHZPMvBK4so1rS5KkcrlXjiRJKobBRJIkFcNgIkmSimEwkSRJxej53YXdwE+SpHL0dDCZ3sBveq+c6Q38AMOJJEkt6OlHOW7gJ0lSWXo6mLiBnyRJZenpYOIGfpIklaWng4kb+EmSVJaeHvzqBn6SJJWlp4MJuIGfJEkl6elHOZIkqSwGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxTCYSJKkYhhMJElSMQwmkiSpGAYTSZJUDIOJJEkqhsFEkiQVw2AiSZKK0dO7Cw+NjLFxyyg7JyZZuaKf9evWuNOwJEkt6tlgMjQyxobNO5ic2g3A2MQkGzbvADCcSJLUkp59lLNxy+jzoWTa5NRuNm4ZbakiSZLUs8Fk58TkfrVLkqT69WwwWbmif7/aJUlS/Xo2mKxft4b+5X0vaOtf3sf6dWtaqkiSJPXs4NfpAa7OypEkqRw9G0ygE04MIpIklaNnH+VIkqTyGEwkSVIxDCaSJKkYBhNJklQMg4kkSSqGwUSSJBXDYCJJkophMJEkScVoJZhExIqIuCkiHoyIByLip9uoQ5IklaWtlV//F/D5zDw/Ig4CDm2pDkmSVJDGg0lE/Djws8DFAJn5DPBM03VIkqTytPEo55XAOPBnETESEddGxGEt1CFJkgrTRjBZBpwKXJ2Za4HvA1fMPikiLomI4YgYHh8fb7pGSZLUgr0Gk4g4cq4/87jmY8BjmXlPdXwTnaDyApm5KTMHM3NwYGBgHpeTJEmLxVxjTLYCCQTwcuCp6vUK4FHghAO5YGZ+MyK+ERFrMnMUOBO4/0A+S5IkLS17DSaZeQJARPwxcGtm/nV1/A7grfO87n8Gbqhm5DwMvHeenydJkpaAbmbl/FRmvn/6IDP/JiJ+ez4XzcztwOB8PkOSJC093QSTb0fEh4BP0nm0cxHwRK1VSZKkntTNrJwLgQHglurPQNUmSZK0oOa8YxIRfcCGzLy0oXokSVIPm/OOSWbuBt7YUC2SJKnHdTPGZCQibgU+Q2cxNAAyc3NtVUmSpJ7UTTA5ks5g1zNmtCVgMJEkSQtqn8EkM11jRJIkNWKfwSQiDgHeB7wWOGS6PTP/bY11SZKkHtTNdOFPAD8BrAP+DjgO+F6dRUmSpN7UTTA5MTN/E/h+Zl4PnA28rt6yJElSL+ommExVPyci4mTgpcDq2iqSJEk9q5tZOZsi4gjgN4FbgcOr15IkSQuqm1k511Yv/w54Zb3lSJKkXtbNrJyvAXcDdwF3Zub9tVclSZJ6UjdjTF4D/AlwFPB7EfFwRNxSb1mSJKkXdRNMdtMZALsbeA74FrCrzqIkSVJv6mbw63eBHcBHgWsy84l6S5IkSb2qmzsmFwJ3Av8R+FREfDgizqy3LEmS1Iu6mZXzWeCzEfFq4B3AZcAHgf6aa6vd0MgYG7eMsnNikpUr+lm/bg3nrV3VdlmSJPWsbmbl3AycAjxEZ2bOu4F7aq6rdkMjY2zYvIPJqd0AjE1MsmHzDgDDiSRJLelmjMnvANsyc3fdxTRp45bR50PJtMmp3WzcMmowkSSpJd2MMbkP2BARmwAi4qSIOKfesuq3c2Jyv9olSVL9ugkmfwY8A/xMdfwY8JHaKmrIyhV7HiKzt3ZJklS/boLJqzLzd6k288vMSSBqraoB69etoX953wva+pf3sX7dmpYqkiRJ3YwxeSYi+oEEiIhXAT+staoGTI8jcVaOJEnl6CaYXAl8Hjg+Im4ATgcurrOoppy3dpVBRJKkgnSzjsltEbENOI3OI5xLM/PbtVcmSZJ6TjdjTMjMJzLz/2Tm54CjIuKamuuSJEk9aK/BJCJeHxFfiIh7I+IjEXFMtdja7cD9zZUoSZJ6xVx3TK4B/gL4RWAc2AY8DJyYmR9roDZJktRj5hpjcnBmfrx6PRoRHwCuWGorwEqSpHLMFUwOiYi1/GjNkqeB10dEAGTmtrqLkyRJvWWuYPI48NEZx9+ccZzAGXUVJUmSetNeg0lmvqXJQiRJkrqaLixJktQEg4kkSSqGwUSSJBVjn0vSR8Spe2j+DvD1zHx24UuSJEm9qptN/P4IOBX4Cp2pwydXr4+KiPdn5hdqrE+SJPWQbh7lPAKszczBzHwjsBa4F3gr8Ls11iZJknpMN8Hk1Zl53/RBZt5PJ6g8XF9ZkiSpF3XzKGc0Iq4GPlUd/zLw/yLiYGCqtsokSVLP6eaOycXAQ8BlwOV0NvK7mE4oOeBF2CKiLyJGIuJzB/oZkiRpaenmjslZwB9k5v/cw3tPz+PalwIPAD8+j8+QJElLSDd3TN5J59HNJyLi7IjoJszMKSKOA84Grp3vZ0mSpKVjn8EkM98LnAh8BvgV4GsRMd9A8fvAB4Hn5vk5kiRpCelq5dfMnAL+hs4A2K3AuQd6wYg4B9iVmVv3cd4lETEcEcPj4+MHejlJkrSI7DOYRMRZEfFxOgNgz6fz+OXYeVzzdOCdEfEInaBzRkR8cvZJmbmpWjtlcGBgYB6XkyRJi0U340UuphMg/kNm/nC+F8zMDcAGgIh4M/CBzLxovp8rSZIWv30Gk8y8YOZxRJwO/Epm/lptVUmSpJ7U1QybiDiFzsDXXwL+Cdi8EBfPzC8CX1yIz5IkSYvfXoNJRPwL4ALgQuAJ4EYgMvOAF1WTJEmay1x3TB4E7gJ+PjMfAoiIyxupSpIk9aS5ZuX8IvBN4I6IuCYizgSimbIkSVIv2mswycxbMvOXgVfTGQdyOXBMRFwdEW9vqD5JktRDuln59fuZeUNmngMcB2wHrqi9MkmS1HO6Wvl1WmY+mZl/kpln1FWQJEnqXfsVTCRJkupkMJEkScUwmEiSpGIYTCRJUjEMJpIkqRgGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxTCYSJKkYhhMJElSMQwmkiSpGAYTSZJUDIOJJEkqhsFEkiQVw2AiSZKKYTCRJEnFMJhIkqRiGEwkSVIxDCaSJKkYBhNJklQMg4kkSSqGwUSSJBXDYCJJkophMJEkScUwmEiSpGIYTCRJUjEMJpIkqRgGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxTCYSJKkYhhMJElSMRoPJhFxfETcEREPRMR9EXFp0zVIkqQyLWvhms8Cv5GZ2yLix4CtEXFbZt7fQi2SJKkgjd8xyczHM3Nb9fp7wAPAqqbrkCRJ5Wl1jElErAbWAve0WYckSSpDa8EkIg4HbgYuy8zv7uH9SyJiOCKGx8fHmy9QkiQ1rpVgEhHL6YSSGzJz857OycxNmTmYmYMDAwPNFihJklrRxqycAP4UeCAzP9r09SVJUrnauGNyOvCrwBkRsb3683Mt1CFJkgrT+HThzPx7IJq+riRJKp8rv0qSpGIYTCRJUjEMJpIkqRgGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxTCYSJKkYhhMJElSMQwmkiSpGAYTSZJUDIOJJEkqhsFEkiQVw2AiSZKKYTCRJEnFMJhIkqRiGEwkSVIxDCaSJKkYBhNJklQMg4kkSSqGwUSSJBXDYCJJkophMJEkScUwmEiSpGIYTCRJUjEMJpIkqRgGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxTCYSJKkYhhMJElSMQwmkiSpGAYTSZJUDIOJJEkqhsFEkiQVw2AiSZKKYTCRJEnFMJhIkqRiGEwkSVIxWgkmEXFWRIxGxEMRcUUbNUiSpPIsa/qCEdEH/CHwNuAx4EsRcWtm3t9kHUMjY2zcMsrOiUlWruhn/bo1nLd2VZMlSJKkWdq4Y/Im4KHMfDgznwE+BZzbZAFDI2Ns2LyDsYlJEhibmGTD5h0MjYw1WYYkSZqljWCyCvjGjOPHqrbGbNwyyuTU7he0TU7tZuOW0SbLkCRJs7QRTGIPbfmikyIuiYjhiBgeHx9f0AJ2TkzuV7skSWpGG8HkMeD4GcfHATtnn5SZmzJzMDMHBwYGFrSAlSv696tdkiQ1o41g8iXgpIg4ISIOAi4Abm2ygPXr1tC/vO8Fbf3L+1i/bk2TZUiSpFkan5WTmc9GxH8CtgB9wHWZeV+TNUzPvnFWjiRJZYnMFw3vKM7g4GAODw+3XYYkSVoAEbE1Mwf39J4rv0qSpGIYTCRJUjEMJpIkqRgGE0mSVAyDiSRJKobBRJIkFcNgIkmSimEwkSRJxVgUC6xFxDjw9Zo+/mjg2zV9tl7Ivm6Ofd0s+7s59nVz6uzrV2TmHjfCWxTBpE4RMby31ee0sOzr5tjXzbK/m2NfN6etvvZRjiRJKobBRJIkFcNgApvaLqCH2NfNsa+bZX83x75uTit93fNjTCRJUjm8YyJJkorRs8EkIs6KiNGIeCgirmi7nqUgIq6LiF0Rce+MtiMj4raI+Gr184iqPSLif1f9/5WIOLW9yhefiDg+Iu6IiAci4r6IuLRqt78XWEQcEhH/NyK+XPX1h6v2EyLinqqvb4yIg6r2g6vjh6r3V7dZ/2IUEX0RMRIRn6uO7euaRMQjEbEjIrZHxHDV1ur3SE8Gk4joA/4QeAfwGuDCiHhNu1UtCR8HzprVdgVwe2aeBNxeHUOn70+q/lwCXN1QjUvFs8BvZOZPAqcBv1b9f9j+Xng/BM7IzDcApwBnRcRpwP8APlb19VPA+6rz3wc8lZknAh+rztP+uRR4YMaxfV2vt2TmKTOmBrf6PdKTwQR4E/BQZj6cmc8AnwLObbmmRS8z7wSenNV8LnB99fp64LwZ7X+eHXcDKyLi2GYqXfwy8/HM3Fa9/h6dL/FV2N8Lruqzp6vD5dWfBM4AbqraZ/f19N/BTcCZERENlbvoRcRxwNnAtdVxYF83rdXvkV4NJquAb8w4fqxq08I7JjMfh84/psDLqnb/DhZIdft6LXAP9nctqkcL24FdwG3A14CJzHy2OmVmfz7f19X73wGOarbiRe33gQ8Cz1XHR2Ff1ymBL0TE1oi4pGpr9Xtk2UJ/4CKxp0Tt9KRm+XewACLicOBm4LLM/O4cvyza3/OQmbuBUyJiBXAL8JN7Oq36aV8foIg4B9iVmVsj4s3TzXs41b5eOKdn5s6IeBlwW0Q8OMe5jfR3r94xeQw4fsbxccDOlmpZ6r41fauv+rmravfvYJ4iYjmdUHJDZm6umu3vGmXmBPBFOuN6VkTE9C93M/vz+b6u3n8pL37EqT07HXhnRDxC5xH7GXTuoNjXNcnMndXPXXRC95to+XukV4PJl4CTqpHeBwEXALe2XNNSdSvwnur1e4DPzmh/dzXK+zTgO9O3DrVv1XP0PwUeyMyPznjL/l5gETFQ3SkhIvqBt9IZ03MHcH512uy+nv47OB/423TBqK5k5obMPC4zV9P5Xv7bzHwX9nUtIuKwiPix6dfA24F7afl7pGcXWIuIn6OTxPuA6zLzqpZLWvQi4i+BN9PZkfJbwJXAEPBp4OXAo8C/ycwnq39Y/4DOLJ4fAO/NzOE26l6MIuJfAncBO/jRs/j/Qmecif29gCLi9XQGAPbR+WXu05n5XyPilXR+qz8SGAEuyswfRsQhwCfojPt5ErggMx9up/rFq3qU84HMPMe+rkfVr7dUh8uAv8jMqyLiKFr8HunZYCJJksrTq49yJElSgQwmkiSpGAYTSZJUDIOJJEkqhsFEkiQVw2AiqTYRsbvatfTeiPhMRBxatT+9r/9WUm8ymEiq02S1a+nJwDPA+9suSFLZDCaSmnIXcOLMhog4PCJuj4htEbEjIs6t2n87Ii6dcd5VEfHrEXFsRNw54y7Mv2r4f4OkmrnAmqTaRMTTmXl4tY/JzcDnM/PqWe2HVhsQHg3cDZwEvALYnJmnRsRLgK/S2cPjYuCQanXKvuq//V4r/+Mk1aJXdxeW1Iz+iNhevb6Lzv4+MwXw3yLiZ+ksrb+Kzpbrj0TEExGxFjgGGMnMJyLiS8B11QaGQ5m5HUlLisFEUp0mM/OUOd5/FzAAvDEzp6pdZQ+p3ruWzh2SnwCuA8jMO6sQczbwiYjYmJl/XlfxkprnGBNJbXopsKsKJW+h8whn2i10Ngv7KWALQES8ojr/Gjp3X05tuF5JNfOOiaQ23QD8VUQMA9uBB6ffyMxnIuIOYCIzd1fNbwbWR8QU8DTw7obrlVQzB79KKlI16HUbnS3Xv9p2PZKa4aMcScWJiNcADwG3G0qk3uIdE0mSVAzvmEiSpGIYTCRJUjEMJpIkqRgGE0mSVAyDiSRJKobBRJIkFeP/A49euOmJXY+EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "n = 10\n",
    "probs = np.random.rand(n)\n",
    "record = np.zeros((n,2))\n",
    "\n",
    "fig,ax = plt.subplots(1,1)\n",
    "ax.set_xlabel(\"Plays\")\n",
    "ax.set_ylabel(\"Avg Reward\")\n",
    "fig.set_size_inches(9,5)\n",
    "rewards = [0]\n",
    "for i in range(500):\n",
    "    p = softmax(record[:,1])\n",
    "    choice = np.random.choice(np.arange(n),p=p)\n",
    "    r = get_reward(probs[choice])\n",
    "    record = update_record(record,choice,r)\n",
    "    mean_reward = ((i+1) * rewards[-1] + r)/(i+2)\n",
    "    rewards.append(mean_reward)\n",
    "ax.scatter(np.arange(len(rewards)),rewards)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Section 2.4\n",
    "Working with PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 2, 3],\n",
       "       [4, 5, 6]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy\n",
    " \n",
    "numpy.array([[1, 2, 3], [4, 5, 6]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 2., 3.],\n",
       "        [4., 5., 6.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    " \n",
    "torch.Tensor([[1, 2, 3], [4, 5, 6]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ -51.9402, -103.8803])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.Tensor([2,4]) #input data\n",
    "m = torch.randn(2, requires_grad=True) #parameter 1\n",
    "b = torch.randn(1, requires_grad=True) #parameter 2\n",
    "y = m*x+b #linear model\n",
    "y_known = torch.Tensor([5,9])\n",
    "loss = (torch.sum(y_known - y))**2 #loss function\n",
    "loss.backward() #calculate gradients\n",
    "m.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(10, 150),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(150, 4),\n",
    "    torch.nn.ReLU(),\n",
    ")\n",
    " \n",
    "loss_fn = torch.nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.randn(10,10)\n",
    "y_correct = torch.ones(10,4)\n",
    "for step in range(100):\n",
    "    y_pred = model(x)\n",
    "    loss = loss_fn(y_pred, y_correct)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.nn import Module, Linear\n",
    "from torch.functional import F\n",
    " \n",
    "class MyNet(Module):\n",
    "    def __init__(self):\n",
    "        super(MyNet, self).__init__()\n",
    "        self.fc1 = Linear(784, 50)\n",
    "        self.fc2 = Linear(50, 10)\n",
    " \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        return x\n",
    " \n",
    "model = MyNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0660, 0.0965, 0.0000, 0.0892, 0.0412, 0.1516, 0.1182, 0.0000, 0.0000,\n",
       "        0.0000], grad_fn=<ReluBackward0>)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(torch.randn(784))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Listing 2.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ContextBandit:\n",
    "    def __init__(self, arms=10):\n",
    "        self.arms = arms\n",
    "        self.init_distribution(arms)\n",
    "        self.update_state()\n",
    "        \n",
    "    def init_distribution(self, arms):\n",
    "        self.bandit_matrix = np.random.rand(arms,arms)\n",
    "        \n",
    "    def reward(self, prob):\n",
    "        reward = 0\n",
    "        for i in range(self.arms):\n",
    "            if random.random() < prob:\n",
    "                reward += 1\n",
    "        return reward\n",
    "        \n",
    "    def get_state(self):\n",
    "        return self.state\n",
    "\n",
    "    def update_state(self):\n",
    "        self.state = np.random.randint(0,self.arms)\n",
    "        \n",
    "    def get_reward(self,arm):\n",
    "        return self.reward(self.bandit_matrix[self.get_state()][arm])\n",
    "        \n",
    "    def choose_arm(self, arm):\n",
    "        reward = self.get_reward(arm)\n",
    "        self.update_state()\n",
    "        return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "env = ContextBandit(arms=10)\n",
    "state = env.get_state()\n",
    "reward = env.choose_arm(1)\n",
    "print(state)\n",
    "print(reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    " \n",
    "arms = 10\n",
    "N, D_in, H, D_out = 1, arms, 100, arms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    "    torch.nn.ReLU(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = torch.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = ContextBandit(arms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(N, pos, val=1):\n",
    "    one_hot_vec = np.zeros(N)\n",
    "    one_hot_vec[pos] = val\n",
    "    return one_hot_vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 2.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(env, epochs=10000, learning_rate=1e-3):\n",
    "    cur_state = torch.Tensor(one_hot(arms,env.get_state()))\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    rewards = []\n",
    "    for i in range(epochs):\n",
    "        y_pred = model(cur_state)\n",
    "        av_softmax = softmax(y_pred.data.numpy(), tau=2.0)\n",
    "        av_softmax /= av_softmax.sum()\n",
    "        choice = np.random.choice(arms, p=av_softmax)\n",
    "        cur_reward = env.choose_arm(choice)\n",
    "        one_hot_reward = y_pred.data.numpy().copy()\n",
    "        one_hot_reward[choice] = cur_reward\n",
    "        reward = torch.Tensor(one_hot_reward)\n",
    "        rewards.append(cur_reward)\n",
    "        loss = loss_fn(y_pred, reward)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        cur_state = torch.Tensor(one_hot(arms,env.get_state()))\n",
    "    return np.array(rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = train(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def moving_average(x,step=50,window=500):\n",
    "    num = (x.shape[0] - window) / step\n",
    "    num = int(num)\n",
    "    avg = np.zeros(num)\n",
    "    slider = np.ones(window) / window\n",
    "    start = 0\n",
    "    for i in range(num):\n",
    "        end = start + window\n",
    "        avg[i] = slider @ x[start:end]\n",
    "        start = start + step\n",
    "    return avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1a1e860590>]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXyU1b3H8c8vOwlZyEZIQhL2fQ87KipaQQWsG9bd3lKt1S63rba1rbXtrd5a21q9UtxQ61ZREBVcUNnXsAeSQBISsq9kI3vm3D8yYAgJmcAkM5n83q9XXpmc58nMjyfhm2fOc55zxBiDUkqpns/N0QUopZSyDw10pZRyERroSinlIjTQlVLKRWigK6WUi/Bw1AuHhoaauLg4R728Ukr1SHv27Ck2xoS1tc1hgR4XF0dCQoKjXl4ppXokEclsb5t2uSillIvQQFdKKRehga6UUi5CA10ppVyEBrpSSrkIDXSllHIRGuhKKeUiNNCVUj1aUWUdnxzMQ6cCtzHQReQnInJYRBJF5G0R8Wm13VtE3hWRVBHZKSJxXVGsUkq1ZIzhJ+/u58G39rI9vcTR5Thch4EuIlHAw0C8MWYs4A4sabXbd4GTxpihwN+Ap+xdqFJKtbb2UD5bUotxdxNe2JDm6HIcztYuFw+gj4h4AL5Abqvti4DXrI9XAleKiNinRKWUOtfx4lM88fFhxkQG8NOrhrP5WDGHsssdXZZDdRjoxpgc4GngBJAHlBtjPm+1WxSQZd2/ESgHQlo/l4gsFZEEEUkoKiq62NqVUr1UQkYpi5/fSn2jhaduHM9dM2Px9/Hgxc3pji7NoWzpculH8xn4ICAS8BORO1rv1sa3nnOFwhiz3BgTb4yJDwtrc7IwpZTq0B8+PoK/jwcfPjiHsVGB+Pt4snBCJJ8fyedUXaOjy3MYW7pc5gHHjTFFxpgG4ANgVqt9soGBANZumUCg1J6FKqUUQE5ZDQeyy7l9eiwxIb5n2hdOiKS2wcL6pAIHVudYtgT6CWCGiPha+8WvBJJa7bMGuNv6+CbgK6NjiJRSXeCzxHwArhkbcVb71LhgBgT6sGZ/60t8vYctfeg7ab7QuRc4ZP2e5SLyhIgstO72MhAiIqnAT4FHu6hepVQv92liPiMj/BkU6ndWu5ubcN34AWw6VsTJU/UOqq59qYWVfNnF7x5sGuVijPmdMWakMWasMeZOY0ydMea3xpg11u21xpibjTFDjTHTjDG9+8qEUsrufvzOPm7513Z2Z5aec3Z+2o1Tomm0GP74SetOhO61P6uMv68/SmOT5czXN/zfNr77WgKfH87vstfVO0WVUk7vaEElq/fnknOyhmBfLxZNjGpzv5ERATx0xTDe35vNB3uzAWhospBRfKpb6myyGF7anM7Ny7bx9/XHWLUvh/SiKu58aSdBvp6MiQzgv/9zgONdVI/DlqBTSqn2pORXcji3nPpGCwvGD2D1vhzc3YTVD84mzN/7vN/78BVD2ZFewi8/OES4vw+vbD3OxqNFbPjZXAYG+573ey/G0YJKfvbeAQ5mlzNvVH/yymt49qtjhPv7IAJvf28GANf9cwuvbcvg8YVj7F6DBrpSymkUVtTyvdcTONDiBqGPDuaSUVzN7KGhHYY5gIe7Gy/cPplbl+/gjpd3nmnfnVHaZYHeZDE89NY+iqvq+MeSiSycEMmGlCLuXbGbrNIanrllAtH9ml/7gwdmERvi18EzXhjtclFKOY2PD+ZxILucx64dxZf/fRl/WDyWrakl5JTVcMOkSJufJ6SvN//+7nSmxvXjN9eNxt/bgz2ZJ+1W54GsMn616tCZMe8f7s8hpaCSJxaNZdHEKESEuSPCuHxEGDdMiuKGSd90EQ0O64u7W9fcSK9n6Eopp3Ekr4LQvt781yWDARgS1peU/Ao+P1zA1aPbvhDanohAH967v/mWmQ0phXYLdGMMv1p1iMO5FeSV1fCb60bzzBdHGRsVwPwWF2tFhFfumUp3zoKiZ+hKKadxJLeC0ZEBZ7X9cfE4Nj9yOX7eF37+OSW2HykFlVTUNlxsiXx2uIDDuRXMG9Wfr1OKuOKvG8krr+WX80fh1urMu7untNIzdKWUU6hvtJBaWMUlw0PP2ebt4X5Rzx0fG4wxsP9EGZcOv/BpRywWw9/XH2VwqB/L7pjMB/tyqKpt5KrR/bv0gqutNNCVUk4hraiK+iYLowcEdLxzJ02MCcJNICHz5EUF+n8SskjOr+TZ2ybh4e7GLfED7VjlxdMuF6UuUFZpNUdyKxxdhss4fSzHRNo/0Pt6ezAmMpCPD+RS29Bk8/c9uS6ZT61TDZRXN/C/n6UwNa4f148fYPca7UEDXakLUHqqnpuWbWPx81vZcqzY0eW4hCN5Ffh4ujEotG+XPP8j14wkvfgUT65Ltmn/1MJKlm1M4w8fH6GxycLf1h+lrLqexxeO6fa+cVtpoCvVScYYfvbeAU6eaiA6uA/fez2Bw7m9e2EFe0jKq2BEf/8uG9I3Z1go986OY8W2DBIyOp4M9t3dWUDz7I7PfpXKGzsy+c70GMZEBnZJffagga5UJz3/dSpfJRfy62tH8e7Smbi7CW9sz3R0WXZV19iExdJ9E6YaYzicW8GoLug/b+kX3xpJYB9PVmzLONOWVVrNP9Yf46G393HXK7v47ordfH44n/f35nD16P7EBPvy7JfH8Pfx4L+vGtGl9V0sDXSlOuHTxDye/vwoiydGctfMWML8vbliZDifHyk4MxFTT1fX2MQVT2/kyU9t65qwh/TiU5TXNDApJqhLX6ePlzvfnhzFZ4fzKays5c/rkrjirxv4+5dH2Z91kvKaBpLzK1n6xh5KT9Xznekx3Dc7DoCfXT2Cfn5eXVrfxdJRLkrZ6L2ELH616hATBgbx5I3jz/Sjzh8bwZoDuezKKGXWkHOH3PU0nybmk1NWw5s7MnnoiqEk51dSVdvI5SPDu+w192Q03/QzJbZfl73GabdPj+HVrRncvGw7mSXV3Dwlmp9ePZwBgX2A5j9o/1h/jGOFVVwyLIw5Q0OJCfFl7vCu+/fbiwa6UjZ4Zctxnvj4CLOHhvD8dybj4/nNuOjLRoTh4+nGZ4n53R7o5TUNrN6Xw8SBQUwYaJ+z2ze2ZxLk60lZdQNPf5bCe3uyqa5v4o+Lx3LHjFi7vEZrezJPEuTryeAuuiDa0tBwf6bFBbMro5QH5g7hkWtGnrXd28OdX5zVJlwxsn+X12UPGuhKdWDfiZP8z9okrh7dn+dvn4yn+9k9lb5eHlw2PIy1ifncN2dQl0281NpXyQU89NY+TtU3MSYygE8evuSin/NIbgUJmSd57NpRrD2Ux2vbM+nn68mU2H48tjqRwopafjRvuN0vXCZkljI5pt85d1p2lT/eMJa9mSe5dapzjSO/WNqHrlQ7iqvqeHXrcR58cy/9A3z4y80Tzgnz0+6bPYjqukau+tsmVu7J7vRrHcmt4PaXdlDVwQLHR3IrOJxbTnV9I79elUhUvz7cOzuOw7kVdhkTv3p/Dl7ubtw0JZrvXzYEL3c3nr55Ai/fPZUbJ0fz7Fep3Ldit10vmJ48VU9a0alu6W45bXh/f5ZMi3Ha4YcXSgNdqTbkl9ey8J9b+P1HR/D19uCFOyYT2Mez3f2nDw7hq5/NZUJ0II+vOUxJVR0bjxbx1s4TNr3e27tOsDW1pN3hdCVVddz+0g4WPLuZRc9tZenre8grr+WPi8fx8BXD8HJ34709WRf0b21pf1YZY6MCCPL14ltjItj/u6u4clR/vDzc+OstE3h0/kg2Hi1iS2oxxwoqWfT8VrJKq21+/pr6JjakFJ7VtvdE9/WfuzoNdKVaqapr5J5Xd1Fe08D7D8xi/U8vY3x0x/3T/QN8+PO3x1HT0MTD7+zjv17bza9WHToTWO0xxpxZqf5gdtvj2f+94wTb0kp45JqRzB4aypbUYq4bP4Bpg4Lp5+fFvNHhfLg/l7pG2++CbK3JYjicU37Wv9XX6+xe2XtnxxHi58UbOzL587pkDmSV8fKW4za/xpPrkrjn1d1sTys507Y+qRAPN2GCDcdYnZ8GulKtrNqbTXJ+Jc/fPrnTZ41Dw/1ZMnUgW1NLiAvxI8zfmyc+OnLeLorEnAryymsBOJhdds52Ywyr9+cwY1AID8wdwst3x/OPJRP54+KxZ/ZZMjWG0lP1fOtvm9h4tKhTNZ+WXlTFqfomxkW1f+OMt4c7t04dyPqkAr5KLiTI15OVe7I77Co6/fxvWt+xnP4jsDujlHd2n+A702Po43VxE3ApDXSlzrEtrYSooD5cdoGTOP331SP47pxBvHbfNB65ZiT7s8r46GDuOfvll9eSWXKKL5IKcBO4fEQYB7LLMebs8D+QXc7x4lNnFknwcHdj0cQogny/GRN96fAwXrknHjcRHnpr73nHxNfUN7HwuS1n1tw87fS7g/HR578T8jvTYwAI9/dm2R1TqKpr5P3zXDfIKq3m+a9T+cXKg3h7uHHbtBi+TC7g65RCfv7eAaKC+pwz0kRdmA5HuYjICODdFk2Dgd8aY/7eYp+5wIfA6fdeHxhjnrBjnUp1C4vFsCO9hCtG9r/gC2bBfl785rrRAHx7UhT/2pjGS5uPs3BCJKv25bBqXw6phVVnzsq93N2Ijw1m7ohwvk4pIr+iloziaibFBOHj6c7qfTl4ebhxzbjzL/Bwxcj+1NRbePCtvezLKmNqXHCb+722PYOD2eWsPZTPtydHn2k/lFOOr5c7g8POP3Qwup8vj18/hkGhfswYHMKEgUG8tCWdW6cOPGs4J0Bjk4XvvrabowVVuAn87voxzB8bwco9Wdz76m58vdx55Z6pFzXXufpGh0fRGJMCTAQQEXcgB1jVxq6bjTHX2bc8pbpXSkElJ6sbmDkkxC7P5+Ym3D0rjsdWJ/LGjkweX3OY2BA/pg0KZkJ0EBZjWJeYz92z4ogM8gHgsVWJfJlcyLXjBvCTq4bz/t5s5o0KJ8Cn/Yuyp80ZFoq7m7AhpfCsQM8vr+W3HyZy2YgwXtiQBjRfADXGnPnDdTC7jLGRgTYNSbx7VtyZx49eM5LbXtzBs18eazV+G1buyeZoQRXPf2cy3xrTHw/rKKFfzh9FblkN379siE3rhCrbdPbP4pVAmjHGtSauUMrq9MU6ewU6wA2Tonjq02R+++Fhwvy9Wf3g7LNGzJxebq22oQkPN+HL5ELC/L355FAeG48W4ePpxqPXjLLptQL7NI8Z/zq5iOH9/Vm+KZ2nbhzPn9clsTW1hM+PNF98XTJ1IO/sziK3vJaooD40NFk4nFtxQTcOzRwSwk1Tolm+KZ2FEyMZGdE8H0t1fSN//eIoU2L7sWBcxFnveO6bM6jTr6M61tk+9CXA2+1smykiB0RknYiMuci6lLpgnxzM4+5XdpFRfIrcshr++eUxymtsW3psW1oJMcG+RAX1sVs9ft4e3GpdCOF3149ud/ijj6c7oyMDCPDxYM0PZ7NgXAQWY3j57qnEhNi+Gs7lI8I5klfBz1ce5HBuBYuf38rW1BL+dMNYlt0xhb/ePIHbpjX3g+8/0XwR9tPEfOoaLUyOubChg79eMAoPd+GdXd8MnVy5J5uiyjoeuWaky433dlY2n6GLiBewEPhlG5v3ArHGmCoRWQCsBoa18RxLgaUAMTExF1SwUq3VN1p4f282eeW1hPb14vcfHaHJYlj8f1tpajJU1jVScqp5HuvzqW1oYmd6CQvG2X/xgh/NG8a0QcFcNfr8t5D/5aYJNFkMAwL78Nxtk6msbSTQt+OulpbmjgjjqU+TCfb14qW74/nFyoMMCvPjOy1upKlrbMLL3Y0D2WXMGRbKEx8fYWxUAN8ac2G3uPfz82Lm4JAzI2wsFsNr2zKYEB3I1DgdX95dOtPlMh/Ya4wpaL3BGFPR4vFaEfk/EQk1xhS32m85sBwgPj6+++bmVC5lW2oxH+7P5U83jCWvvJY7Xt5JZkk1ImAMTBgYxP/cMJafvXeQoD6eBPl68u8dmdwzK4640PZvy/9wfw6VdY3cMDnK7jX7+3hy9ZiOV60fEeF/5rGbm3Q6zAFGRvjz43nDuGp0f8ZEBvLJw3OAsxcs9vZofjeQkFHKY6sTKamq49V7pp7p474Qlw0P4+uPjpBZcorMkmrSik7xzC0T9Oy8G3Um0G+jne4WEYkACowxRkSm0dyVU9LWvkpdDGMM/7MuicScCibFBLEhpYiiyuYwmhzTj10ZpUwfHEyAjydrH56DiFBYUcvGo0X85bMUnr99crvP++rWDEZG+DN9UNujQ3oKEeHH84af9XVbJg4MYsW2DPaeKOPn3xrB2POMP7fFZSPC4aMjbEgp4vMj+YT29eJaJ12qzVXZFOgi4gtcBXy/Rdv9AMaYZcBNwAMi0gjUAEtM68G0StnBruOlJOZU4Oflzp/WJlFZ28hPrxp+ZmrXll0ap4MsPMCHu2fF8a+NaeSX1xIR2Dya5K2dJ6isbeC+OYPYdbyU5PxKnvz2uF5zRnnlqHBW7snm8YVjuGlKdMff0IG4EF9ign35y2cpVNU18vj1o/H20JuFupNNgW6MqQZCWrUta/H4OeA5+5am1Lle3nKcfr6ePPedydz+0k7C/b35r0s6HjFxa/xAXtiQxgf7svnB3KHkldfwuzWJNDQZ/r0zk9yyWkL7erNoov27W5zVJcPCOPT41Xb7AyYizB0RxuvbM7lpSvRZQxtV99DR/KrHyC+v5YukAn4wdwizh4byh8VjGRnhf858I22JC/VjWlwwKxOyeeCyIby0+TgWA79fOIYP9uVwzZgI7pszqNfdfm7vdyN3z4rD18uDn1w1rNe803EmGuiqx1ifVIAxnLkF/s5Ojpm+KT6aX6w8yAsb03hr5wkWTYjk7llxeiZpR0PC+vLofL2N31E00BUAH+zNJjKoDzMGX/wNNU0Ww57Mk2w8WkhKfiUVtY1cMjSUW6cNJNzf54Kfd31SAbEhvgzp4Nb09lw7bgBPrkvmfz9Nwd1NuH/ukAuuRSlnpIGuOHmqnp/+5wDQPPfIUzeNb3chh47UNjRx34rdbEsrwdNdGBLWF093N55Zf5RNx4p47/5ZF/S81fWNbEsr4Y7psRf8Vt7P24Otj1xBfkUtHm7CwGDbb9ZRqifQQFccsE7ZOm9Ufz7Yl8OsoaEXNOqhpr6JH7y5h+3pJTx+/WhunBKNv3X+kRc3pfOntUkcya1gdGRAp59787Fi6hstzBt1cQv19vFyZ9B5xqIr1ZPp9Lm92Ok5uvdnlSECf7t1AiMj/Fm2Mc3mJcZ2pJdw34rd/HltEvOe2cjXKUX8cfFY7pk96EyYA9wcH423hxv/3nlh0wB9djgffx8PpvbwMeJKdSUN9F7o7+uPMv1/1jP8sXXsOl7K/qwyhof74+/jyQNzh5BaWHVmEqeOPP91KltSi3lxczp+3u68u3QGt08/92JlkK8XCydEsnpfDhW138yrUlhRyycH8877ByQpr4IP9+dyw6SoC+4KUqo30C6XXiaj+BT/+PIY0+KCqWu08MqW4xzIKjtzQ8614wbwty+O8sj7BymuqmPJ1IF4uLthsRgOZJeRnF/JpcPDiArqQ2FlLVtTi/nB3KH88IqheHu4nbd/+86Zsby3J5t3dp1g6aVD+PeOTJ5cl0xVXSO3xEdz35xBrDuUz+0zYgj39+GNHZkIsHpfDgE+Hvz0quHtPrdSSgO911mxLQMPN+Gft03ipS3HeXFzOsbAxIHNEyh5uLvxyj1T+dWqQzy2OpFlG9OYGhfM5mPFFFfVASACiydGMSLCH4uBxZMiz1nYoC3jo4OYNSSElzYfJybYj8dWJzJnaCgjIvx5ectx/pPQvOrN3hMnuTl+IL9ZnXjme5+6cdxZK/Qopc6lgd6LlNc08J+ELK4fH0l4gA9Lpg5k+aZ0oHlej9MGh/Xl7e/N4IsjBby4OZ2vUwqZMzSUeaP6M3KAP6v25vCvTemIwLioQIaG+7f3kuf4wdyh3PHyTn741l6Ghffl5Xvi8fZovlBZXtOAu5vw5LpktqeVMHFgEE/dOJ7jxVVcPbrjia2U6u000HuRFzelU13fdGZxgcFhfZk5OIQD2WUM73/22G4R4eoxEW3OEPjLBQFEB/vym9WJ3BLfudEws4eGMD46kMSccp6+ecKZuT5OL6xgjGH38VK2p5fwt1snMijU76wZCJVS7RNHzaEVHx9vEhISHPLavVFiTjmLn9/KwgmRPHPrxDPtmSWnyCqtYc6w0E4/Z355Lf0DvDs9Ljz7ZDUnSquZNaTt12xsslBW00BoX12aTKnWRGSPMSa+rW16ht4LFFXW8bP3DtDPz4vfXj/6rG2xIX7EhlzYuOzTsxZ2VnQ/X6L7tX9Tj4e7m4a5UhdAA91FNVkMHx/MJftkDSu2ZVBR08Dyu+L1wqJSLkwD3UVtSCnkR+/sB2B4/7688d1pZxbvVUq5Jg10F5WQeRJPd2H3r+cR2MdTpzJVqhfQQHdRezJPMjoyULtYlOpF9D5qF9TQZOFgdhlTYnS1daV6Ew10F5SUV0Ftg4XJsUEd76yUchka6C5ob+ZJACbrGbpSvYoGugvac6KMAYE+RAb1cXQpSqlupIHuYhqaLGxPK2FyrJ6dK9XbdBjoIjJCRPa3+KgQkR+32kdE5FkRSRWRgyIyuetKVuezLjGf4qo6bprc+RWHlFI9W4fDFo0xKcBEABFxB3KAVa12mw8Ms35MB16wflbd7LVtGcSF+HLZ8DBHl6KU6mad7XK5EkgzxrReR2wR8LpptgMIEpEBdqlQ2exQdjl7Mk9y18w43Nz0RiKlepvOBvoS4O022qOArBZfZ1vbziIiS0UkQUQSioqKOvnSqiNv7sykj6c7N3VySlullGuwOdBFxAtYCLzX1uY22s6Zl9cYs9wYE2+MiQ8L0y4Be6qqa2TNgVyunzCAgBaLMyuleo/OnKHPB/YaY9paPTgbGNji62gg92IKU53zycFcquubuHXqwI53Vkq5pM4E+m203d0CsAa4yzraZQZQbozJu+jqVLsO55Yz/x+bWb0vh9qGJt7alcXQ8L56M5FSvZhNk3OJiC9wFfD9Fm33AxhjlgFrgQVAKlAN3Gv3StVZXtiQRlJeBT9+dz9eK92ob7Lw+4VjdFZFpXoxmwLdGFMNhLRqW9bisQEetG9pqj355bWsS8zn3tlxxAb7kllazeUjwrnkApaRU0q5Dp0+twd6c2cmFmO4d9YgYkLaX8pNKdW76K3/PUx9o4W3d53gypHhGuZKqbNooPcwXyUXUFxVz+3TYx1dilLKyWig9zDv7M4iIsCHS/XWfqVUKxroPUhuWQ0bjxZxS3w07nprv1KqFQ30HmT5pnSMgZvj9eYhpdS5NNB7iLWH8lixLYO7ZsYyMFgvhiqlzqWB3gMUVtTy8/cOMHFgEL++dpSjy1FKOSkNdCdT32hh0fNb+c/ubyav/PfOE1Q3NPHMLRPw9nB3YHVKKWemge5kPtyfw4GsMj4+1DwVTl1jE2/tPMHlI8IZHNbXwdUppZyZBroTMcbw4uZ0APZlnqTJYlh7KI/iqjrunhXn2OKUUk5PA93BLBZD81Q4sD6pkKMFVVw6PIzKukZS8it5Y3smg0P9uGSoztOilDo/ncvFwe5dsZu0oiquGx/Jim3HiQ3x5fcLx3D50xt4e9cJ9p4o47FrR+mSckqpDmmgO1BFbQObjxXh6+XBso1pzBoSwt+XTCSsrzf9A7z5985MPN2FGyads5qfUkqdQwPdgXall2IxsPyuKQT4eDJqQMCZO0DjY4P55FAeV4+OIKSvt4MrVUr1BNqH7kBb04rx9nBjSmw/xkYFnnU7/9S45pWHbtEl5ZRSNtIzdAfanlbC1LjgNseW3zJ1IKH+3lyqi1YopWykZ+gOUlxVR3J+JbOGhrS53dfLg+vGR+qSckopm2mgO8jW1GIAZg3RM3CllH1ooDvIB3tziAjwYVxUoKNLUUq5CA10B8gtq2HTMZ3XXCllXzYFuogEichKEUkWkSQRmdlq+1wRKReR/daP33ZNua7hvYRsQOc1V0rZl62jXP4BfGqMuUlEvIC2JuTebIy5zn6luaaK2gbe3X2C2UNCdV5zpZRddXiGLiIBwKXAywDGmHpjTFlXF+aKyqsbuPOlnRRV1fGDy4c4uhyllIuxpctlMFAEvCoi+0TkJRHxa2O/mSJyQETWiciYtp5IRJaKSIKIJBQVFV1M3T3SHz45QlJeJS/cPkVHtyil7M6WQPcAJgMvGGMmAaeAR1vtsxeINcZMAP4JrG7riYwxy40x8caY+LCw3rVqfZPFsD6pgOvGD2De6P6OLkcp5YJsCfRsINsYs9P69UqaA/4MY0yFMabK+ngt4Ckiegrawr4TJymrbuCKUeGOLkUp5aI6DHRjTD6QJSIjrE1XAkda7iMiEWK9pVFEplmft8TOtfZoXyYX4u4mXDKsd70zUUp1H1tHuTwEvGkd4ZIO3Csi9wMYY5YBNwEPiEgjUAMsMadXbVAAfJ1cSHxsPwL7eDq6FKWUi7Ip0I0x+4H4Vs3LWmx/DnjOjnW5lB3pJSTnV/LL+SMdXYpSyoXpbItdyBjD7z86woptGQwI9GHhxEhHl6SUcmEa6F3oL5+lsGJbBnfOiOWR+SPp662HWynVdTRhushnh/P5vw1p3DYthicWjdFpcJVSXU4n5+oiXycXEuTryR80zJVS3UQDvYsk5pYzLioQD3c9xEqp7qFp0wXqGy2k5FcyOjLA0aUopXoRDfQucLSgkoYmw9hIXbxCKdV9NNC7wOHccgDG6mpESqlupIHeBRJzKujr7UGszneulOpGGuhd4HBuOaMjA3DT5eWUUt1IA93OmiyGI3kV2n+ulOp2Guh2tjujlNoGC5NjgxxdilKql9FAt7N1h/Lw9nDj8hE677lSqntpoNuRxWJYl5jP3BFh+Om8LUqpbqaBbkcJmScprKxjwbgBji5FKdULaaDb0ScHc/HycOPKUbpmqFKq+2mg20lFbQPv783hW2MidJpcpZRDaKDbyds7T1BV18j3Lx3s6FKUUr2UBrod1DdaeHVrBrOGhOjt/koph9FAt4MNKYXkV9TyvUv07Fwp5Tga6D7zDxIAAA5dSURBVHawI70Ubw83Zg8NdXQpSqleTAPdDnZnlDIpJggvDz2cSinHsSmBRCRIRFaKSLKIJInIzFbbRUSeFZFUETkoIpO7plznU1nbwOHccqYNCnF0KUqpXs7W8XX/AD41xtwkIl5A63lh5wPDrB/TgResn13ensyTWAxMHxTs6FKUUr1ch2foIhIAXAq8DGCMqTfGlLXabRHwumm2AwgSkV5xu+TujFI83IRJMToZl1LKsWzpchkMFAGvisg+EXlJRPxa7RMFZLX4OtvadhYRWSoiCSKSUFRUdMFFO5Ndx0sZGxWIr5feTKSUcixbAt0DmAy8YIyZBJwCHm21T1srOZhzGoxZboyJN8bEh4WFdbpYZ3O8+BR7Mk9y6TAd3aKUcjxbAj0byDbG7LR+vZLmgG+9z8AWX0cDuRdfnnPKKq2mocnCvzam4enuxp0z4xxdklJKdXxR1BiTLyJZIjLCGJMCXAkcabXbGuCHIvIOzRdDy40xefYv1/GySquZ+/QGRkb4c7SgkiVTYwjz93Z0WUopZfMol4eAN60jXNKBe0XkfgBjzDJgLbAASAWqgXu7oFankJBZSpPFkFF8CmNgqc7dopRyEjYFujFmPxDfqnlZi+0GeNCOdTmtfSfK8PNyZ8PPL6eoso6Bwa1HcCqllGPo0IxO2neijPHRQYT5e2tXi1LKqei96p1Q29BEUl4FE3XMuVLKCWmgd0JiTjmNFsOkgRroSinno4HeCftONN8gq2foSilnpH3oNqipb+Jfm9J4f282UUF9CPf3cXRJSil1Dj1Dt8GyjWn8ff0xAvt48uN5wxxdjlJKtUnP0DvQ0GTh7V0nmDsijBX3TnN0OUop1S49Q+/AF0cKKKys484ZsY4uRSmlzksDvQNvbM8kKqgPc0eEO7oUpZQ6Lw3080jIKGV7egl3zYzF3a2tCSWVUsp5aKC3wxjDU58mE+7vzV06m6JSqgfQQG/BYjE0NFmobWji9e2Z7M44yUNXDqOPl7ujS1NKqQ716lEuG1IKefrzFH54+VCuGTuA772ewJfJhXi6Cw1NhnFRgdwaP7DjJ1JKKSfQKwO99FQ9T3x0mNX7m9fgeGtXFjOHhLLhaBGXDAtl9IAA5gwLZdaQUO07V0r1GL0q0DcfK+I/CdlsOlpEdX0jD185jJOn6nk3IYv1RwposhgevnIYU+OCHV2qUkp1Wq8J9IraBr7/xh58vdy5bHgYD14+lBER/mw5VswbOzJ55ouj+Pt46MRbSqkey+UD3WIxuLkJq/bmUF3fxDtLZzA++pvQnjqoH75e7uSU1bBgXAQe7nqdWCnVM7l0oJdV13P13zZx1ej+7DxeyvjowLPCHMDbw51ZQ0JZn1TAZcPDHFSpUkpdPJc+Hf3oYB6FlXW8ufMEqYVV3NHO7fsLxkXg7eGmd4MqpXo0lz5Df39PNiMj/Ll9egyfHyng+vGRbe53w6QorhzZn0Bfz26uUCml7MdlAz2tqIr9WWX8asFI7pwZx53nudtTRDTMlVI9nk2BLiIZQCXQBDQaY+JbbZ8LfAgctzZ9YIx5wn5ldt6qvTm4CSyeGOXIMpRSqtt05gz9cmNM8Xm2bzbGXHexBdnL+qQCpg0KJjxAVxdSSvUOLnlRtLiqjuT8Si4ZpqNWlFK9h62BboDPRWSPiCxtZ5+ZInJARNaJyJi2dhCRpSKSICIJRUVFF1SwLballQAwZ2hol72GUko5G1u7XGYbY3JFJBz4QkSSjTGbWmzfC8QaY6pEZAGwGjhn8U1jzHJgOUB8fLy5yNrbtfVYMQE+HoyNCuyql1BKKadj0xm6MSbX+rkQWAVMa7W9whhTZX28FvAUEYecHhtj2JJarBNrKaV6nQ4DXUT8RMT/9GPgaiCx1T4RIiLWx9Osz1ti/3I7lllSTU5ZDbOHaXeLUqp3saXLpT+wyprXHsBbxphPReR+AGPMMuAm4AERaQRqgCXGmC7rUjmfhMyTAMwYpDMmKqV6lw4D3RiTDkxoo31Zi8fPAc/Zt7QLczC7DD8vd4aE9XV0KUop1a1cbtjiwexyxkYF4qb950qpXsalAr2hycKRvArGR+voFqVU7+NSgX60oJL6RgvjonWRCqVU7+NSgX4ouxyA8Tr+XCnVC7lUoB/MKSfAx4PYEF9Hl6KUUt3OtQI9u4zx0UFYh1gqpVSv4jKBnltWQ2JOBdN0/LlSqpdymUBfcyAXgEUT216VSCmlXJ3LBPrqfTlMjgkiNsTP0aUopZRDuESgJ+VVkJxfyQ2TdHUipVTv5RKB/mliPm4C17azCLRSSvUGLhHoyfkVxIX4Eezn5ehSlFLKYVwi0I8VVDGsv07GpZTq3Xp8oNc2NJFRcorh/f0dXYpSSjlUjw/048WnsBgYpoGulOrlenygHy2oBGC4drkopXq5Hh/oxwqqcHcTBoXq+HOlVO/W4wP9aEElsSG+eHu4O7oUpZRyqB4f6McKqxgerv3nSinVowO9tqGJzJJT2n+ulFL08EBPLazSES5KKWVlU6CLSIaIHBKR/SKS0MZ2EZFnRSRVRA6KyGT7l3qupLwKAEYNCOiOl1NKKafm0Yl9LzfGFLezbT4wzPoxHXjB+rlLJedX4u3hpiNclFIK+3W5LAJeN812AEEiMsBOz92upLwKRkT44+6mKxQppZStgW6Az0Vkj4gsbWN7FJDV4utsa1uXMcaQlFfBqAjtblFKKbC9y2W2MSZXRMKBL0Qk2RizqcX2tk6RTesG6x+DpQAxMTGdLraloso6TlY3MHKAXhBVSimw8QzdGJNr/VwIrAKmtdolGxjY4utoILeN51lujIk3xsSHhYVdWMVWR/SCqFJKnaXDQBcRPxHxP/0YuBpIbLXbGuAu62iXGUC5MSbP7tW2kJzfPIeLdrkopVQzW7pc+gOrROT0/m8ZYz4VkfsBjDHLgLXAAiAVqAbu7Zpyv5GUV0FkoA+Bvp5d/VJKKdUjdBjoxph0YEIb7ctaPDbAg/Yt7fwO51YwOlLPzpVS6rQeeadoVV0jaUVVjI8OcnQpSinlNHpkoCfmlGMMjIsOdHQpSinlNHpkoB/KLgdgXJQGulJKndYjA/1gTjlRQX0I7evt6FKUUspp9MhAP5RdpmfnSinVSo8L9PLqBjJKqrX/XCmlWulxgZ6Y29x/PkFHuCil1Fl6XKB7e7gxb1Q4Y6N0DLpSSrXUmfnQnUJ8XDAvxQU7ugyllHI6Pe4MXSmlVNs00JVSykVooCullIvQQFdKKRehga6UUi5CA10ppVyEBrpSSrkIDXSllHIR0rzYkANeWKQIyLzAbw8Fiu1YTldw9hq1vovj7PWB89eo9V2YWGNMWFsbHBboF0NEEowx8Y6u43ycvUat7+I4e33g/DVqffanXS5KKeUiNNCVUspF9NRAX+7oAmzg7DVqfRfH2esD569R67OzHtmHrpRS6lw99QxdKaVUKxroSinlInpcoIvINSKSIiKpIvKoE9QzUES+FpEkETksIj+ytj8uIjkist/6scCBNWaIyCFrHQnWtmAR+UJEjlk/93NgfSNaHKf9IlIhIj925DEUkVdEpFBEElu0tXnMpNmz1t/JgyIy2UH1/UVEkq01rBKRIGt7nIjUtDiOy7q6vvPU2O7PVER+aT2GKSLyLQfV926L2jJEZL+13SHHsNOMMT3mA3AH0oDBgBdwABjt4JoGAJOtj/2Bo8Bo4HHgZ44+Zta6MoDQVm3/Czxqffwo8JSj62zxM84HYh15DIFLgclAYkfHDFgArAMEmAHsdFB9VwMe1sdPtagvruV+Dj6Gbf5Mrf9nDgDewCDr/3P37q6v1fa/Ar915DHs7EdPO0OfBqQaY9KNMfXAO8AiRxZkjMkzxuy1Pq4EkoAoR9Zko0XAa9bHrwGLHVhLS1cCacaYC72L2C6MMZuA0lbN7R2zRcDrptkOIEhEBnR3fcaYz40xjdYvdwDRXVlDR9o5hu1ZBLxjjKkzxhwHUmn+/95lzlefiAhwC/B2V9Zgbz0t0KOArBZfZ+NE4SkiccAkYKe16YfWt7+vOLJLAzDA5yKyR0SWWtv6G2PyoPmPEhDusOrOtoSz/xM5yzGE9o+ZM/5e3kfzu4bTBonIPhHZKCKXOKooq7Z+ps52DC8BCowxx1q0OdMxbFNPC3Rpo80pxl2KSF/gfeDHxpgK4AVgCDARyKP57ZujzDbGTAbmAw+KyKUOrKVdIuIFLATeszY50zE8H6f6vRSRXwONwJvWpjwgxhgzCfgp8JaIBDiovPZ+pk51DIHbOPvEwpmOYbt6WqBnAwNbfB0N5DqoljNExJPmMH/TGPMBgDGmwBjTZIyxAC/SxW8fz8cYk2v9XAisstZScLpbwPq50FH1tTAf2GuMKQDnOoZW7R0zp/m9FJG7geuA242189fajVFifbyH5v7p4Y6o7zw/U2c6hh7At4F3T7c50zE8n54W6LuBYSIyyHo2twRY48iCrH1tLwNJxphnWrS37EO9AUhs/b3dQUT8RMT/9GOaL5wl0nzc7rbudjfwoSPqa+WssyJnOYYttHfM1gB3WUe7zADKT3fNdCcRuQZ4BFhojKlu0R4mIu7Wx4OBYUB6d9dnff32fqZrgCUi4i0ig2iucVd312c1D0g2xmSfbnCmY3hejr4q29kPmkcUHKX5L+SvnaCeOTS/NTwI7Ld+LADeAA5Z29cAAxxU32CaRw8cAA6fPmZACPAlcMz6OdjBx9EXKAECW7Q57BjS/IclD2ig+ezxu+0dM5q7C563/k4eAuIdVF8qzf3Qp38Pl1n3vdH6sz8A7AWud+AxbPdnCvzaegxTgPmOqM/avgK4v9W+DjmGnf3QW/+VUspF9LQuF6WUUu3QQFdKKRehga6UUi5CA10ppVyEBrpSSrkIDXSllHIRGuhKKeUi/h/87L30Fz+hhwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(moving_average(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
